# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import csv

# useful for handling different item types with a single interface
from itemadapter import ItemAdapter

import os


class HourseScrapyPipeline:
    def open_spider(self, spider):
        self.dir_path = './html2'
        if not os.path.isdir(self.dir_path):
            os.makedirs(self.dir_path)
        with open(r'E:\BaiduNetdiskDownload\GAME\OCTOPATH2\hourse_spider\hourse_scrapy\cityUrl.csv', mode="r", newline='') as f:
            reader = csv.reader(f)
            for idx,row in enumerate(reader):
                if idx == 0:
                    continue
                city = row[0]
                if not os.path.isdir(os.path.join(self.dir_path,city)):
                    os.makedirs(os.path.join(self.dir_path,city))
    def process_item(self, item, spider):
        filename = item['filename']
        html = item['html']
        city = item['city']
        filepath = os.path.join(self.dir_path, city , filename)
        with open(filepath, 'wb') as f:
            f.write(html)
        return item
